{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VGG13 CNN in TensorFlow\n",
    "\n",
    "### In this project we are going to:\n",
    "<br/>\n",
    "#### 1. Prepare the environment (e.g., importing the required libraries)\n",
    "#### 2. Load and preprocess a dataset to train on \n",
    "#### 3. Implement a fully functioning ConvNet using TensorFlow\n",
    "#### 4. Train the implemented model on the prepared dataset\n",
    "#### 5. Analyze the results\n",
    "<br/>\n",
    "<br/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **1 - Preparing the environment**\n",
    "\n",
    "As usual, we will start by loading in the packages. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "# Imports\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "\n",
    "%matplotlib inline\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **2 - Load and preprocess the dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/amirhf/.local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST-data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST-data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Loading the data (signs)\n",
    "mnist = tf.contrib.learn.datasets.load_dataset(\"mnist\")\n",
    "train_data = mnist.train.images  # Returns np.array\n",
    "train_labels = np.asarray(mnist.train.labels, dtype=np.int32)\n",
    "eval_data = mnist.test.images  # Returns np.array\n",
    "eval_labels = np.asarray(mnist.test.labels, dtype=np.int32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As a reminder, the MNIST dataset is a collection of images representing numbers from 0 to 9.\n",
    "\n",
    "<img src=\"images/mnist.jpg\" style=\"width:800px;height:800px;\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y = 0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADr1JREFUeJzt3X+QVfV5x/HPI6yAK6YQDN2CCkHSSGwCmRU1IY0ZE4ImM5hJh5GaBJtMsBYancRpif0jpn90nI6/yNRJuokk6ERNpv6iE/NDGRNiVMriUPkhBiQ4QmA3iqkEKj+Wp3/sIV1x7/de7j3nnrv7vF8zO3vvec655/HIZ8+953vv/Zq7C0A8p5TdAIByEH4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0GNbObOTrVRPlrtzdwlEMobOqDDfshqWbeh8JvZPEnLJY2Q9B13vzm1/mi160K7tJFdAkhY66trXrfup/1mNkLSnZIukzRD0kIzm1Hv4wForkZe88+WtN3dd7j7YUn3S5qfT1sAitZI+CdJennA/V3Zsjcxs8Vm1m1m3Ud0qIHdAchT4Vf73b3L3TvdvbNNo4reHYAaNRL+3ZLOGnB/crYMwBDQSPjXSZpuZlPN7FRJV0palU9bAIpW91Cfux81s6WSfqr+ob4V7r45t84AFKqhcX53f1TSozn1AqCJeHsvEBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0E1dYputKCL3pss/+a69GzPv/7wymT93J9fXbE27a83JLdFsTjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQDY3zm9lOSfsl9Uk66u6deTSF/Oy9/gPJ+r8sXZGszx1zIFk/4un9L599f8XaN/Tu9MZV9Px9+r/tz+7dWrHW9+q+hvY9HOTxJp+PuPsrOTwOgCbiaT8QVKPhd0k/M7P1ZrY4j4YANEejT/vnuPtuM3uHpMfMbKu7rxm4QvZHYbEkjdZpDe4OQF4aOvO7++7sd6+khyTNHmSdLnfvdPfONo1qZHcAclR3+M2s3czGHr8taa6kTXk1BqBYjTztnyjpITM7/jj3uvtPcukKQOHqDr+775D0vhx7QQU2Kv1y6bUF769YW3PDrcltT7NT6+qpGXZ9NT2Ov27JHcn6D5dMrlj7xh2fTm575reeTtaHA4b6gKAIPxAU4QeCIvxAUIQfCIrwA0Hx1d1DwI6bKg/lSdLmz/1bolrsUN63fv/OZP3f7/lExdokPZXc9tDbjyXrbTYiWb9q7J6KtQuW3Zbc9rP6crI+HIYCOfMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM87eAah/ZbZ/xWpM6easfHxybrD/wD3OT9Uk/So/ll+Vdben3P9z/1VuS9Y/Puj79+NesO+memo0zPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExTh/E9jI9GF+8Z/Tn9ff0pn6vH5jFr98SbLe++n0OP+o3cWNZ0/50eFk/b3nXJ2sr7/4roq1at8FMHXk6GT9jK1tyfpQwJkfCIrwA0ERfiAowg8ERfiBoAg/EBThB4KqOs5vZiskfVJSr7ufny0bL+kHkqZI2ilpgbuX96HzFnfoo7OS9S2fKW4c/7rffjBZ7/lEery679Xf5tnOSRnxxLPJ+tlPpLd/6IWOirUFp/fW09KwUsuZ/3uS5p2wbJmk1e4+XdLq7D6AIaRq+N19jaR9JyyeL2lldnulpCty7gtAwep9zT/R3Y/PhbRX0sSc+gHQJA1f8HN3l+SV6ma22My6zaz7iA41ujsAOak3/D1m1iFJ2e+KV0/cvcvdO929s03pL6oE0Dz1hn+VpEXZ7UWSHsmnHQDNUjX8ZnafpKcl/bmZ7TKzL0i6WdLHzGybpI9m9wEMIVXH+d19YYXSpTn3MmT1fOkDyfrfXftwoftPjeX/5sPpv+/HDp44kIMoeIcfEBThB4Ii/EBQhB8IivADQRF+ICi+urtGp7zvvIq1m79U+SuiJenSMQcb2ne1r9dOfSx3OA/l2az3JOtT2tIfCU7ZfiT9VvS37Tha92O3Cs78QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4/w1+tA9lceMGx3Hr2bdw3+RrE969alC99+qXrj2tGR99qiK3y5X1U8PzEjWxzzyX3U/dqvgzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHOn3nlmouT9WvH3Zqopmci2tP3v8n6l19Kz3N69oM9yXpfsjp0jZx6TrL+i3m3V3mEMXXv+8l951ZZ45W6H7tVcOYHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaCqjvOb2QpJn5TU6+7nZ8tukvRFSb/LVrvR3R8tqslm2J8eUtbpp6TH8lNu6f1Iet8fqjZmPPTHlOvxwpKOZL1jRP3j+K8deyNZ37t8WrLePgz+n9Ry5v+epHmDLL/d3WdmP0M6+EBEVcPv7mskDd9pX4CgGnnNv9TMnjOzFWY2LreOADRFveH/pqRpkmZK2iOp4hvfzWyxmXWbWfcRpec/A9A8dYXf3Xvcvc/dj0n6tqTZiXW73L3T3TvbqnwABkDz1BV+Mxt4GfZTkjbl0w6AZqllqO8+SZdImmBmuyR9TdIlZjZTkkvaKemaAnsEUICq4Xf3hYMsTk9Ijzf5yeOdyfpUPd2kTlqMWbLsI4rb9Q27LkvW2/9jbXE7bxG8ww8IivADQRF+ICjCDwRF+IGgCD8QFF/d3QQdvxquX67dmP+56sJkfeuCOwvb91O/Sk/BPU3PFLbvVsGZHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpy/Cc65cWuy3vOfTWqkACMnT0rWty05u2Jt7WdS055L1aY+r+a+/RMr1t713deS20Z4ZwZnfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+JpjzJ9uT9YenX5Ss923bkWc7bzLivOnJ+rZFE5L1O/7qu8n63DEHEtViZ3BauWR+xdrIzesL3fdQwJkfCIrwA0ERfiAowg8ERfiBoAg/EBThB4KqOs5vZmdJulvSREkuqcvdl5vZeEk/kDRF0k5JC9w9/SHpFjb9O3uS9a9fPrNi7Wtnbkhu+zdnvJysj1h1LFnfeHByst6Ime2/SNavGps+LkVadWBcsn7D41cm6+9+ZnPFWvqIx1DLmf+opK+4+wxJF0laYmYzJC2TtNrdp0tand0HMERUDb+773H3Z7Pb+yU9L2mSpPmSVmarrZR0RVFNAsjfSb3mN7MpkmZJWitporsff064V/0vCwAMETWH38xOl/SApOvd/fWBNXd39V8PGGy7xWbWbWbdR3SooWYB5Kem8JtZm/qD/313fzBb3GNmHVm9Q1LvYNu6e5e7d7p7Z1vBH+QAULuq4Tczk3SXpOfd/bYBpVWSFmW3F0l6JP/2ABTF+p+xJ1YwmyPpl5I26v9HSG5U/+v+H0o6W9JL6h/q25d6rDNsvF9olzbacyn2ff7iirUff/2W5LZvO2V03u0MGQf9cMXanfsqD59K0prPX5Cse/emunoaztb6ar3u+6yWdauO87v7k5IqPdjQTDIA3uEHREX4gaAIPxAU4QeCIvxAUIQfCKrqOH+ehvI4f8q0delx/L898+fJ+nltbTl201x3/n5asn7P8ssq1iZ0PZ13O+GdzDg/Z34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIopunPw4gVvJOvLzl2Y3v7qP03WPz6vO1m/teOZirX33L00ua31JctVTbv31WR9whbG8lsVZ34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrP8wPDCJ/nB1AV4QeCIvxAUIQfCIrwA0ERfiAowg8EVTX8ZnaWmT1hZlvMbLOZXZctv8nMdpvZhuzn8uLbBZCXWr7M46ikr7j7s2Y2VtJ6M3ssq93u7rcU1x6AolQNv7vvkbQnu73fzJ6XNKnoxgAU66Re85vZFEmzJK3NFi01s+fMbIWZjauwzWIz6zaz7iM61FCzAPJTc/jN7HRJD0i63t1fl/RNSdMkzVT/M4NbB9vO3bvcvdPdO9s0KoeWAeShpvCbWZv6g/99d39Qkty9x9373P2YpG9Lml1cmwDyVsvVfpN0l6Tn3f22Acs7Bqz2KUmb8m8PQFFqudr/QUmflbTRzDZky26UtNDMZkpySTslXVNIhwAKUcvV/iclDfb54EfzbwdAs/AOPyAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFBNnaLbzH4n6aUBiyZIeqVpDZycVu2tVfuS6K1eefZ2jrufWcuKTQ3/W3Zu1u3unaU1kNCqvbVqXxK91aus3njaDwRF+IGgyg5/V8n7T2nV3lq1L4ne6lVKb6W+5gdQnrLP/ABKUkr4zWyemb1gZtvNbFkZPVRiZjvNbGM283B3yb2sMLNeM9s0YNl4M3vMzLZlvwedJq2k3lpi5ubEzNKlHrtWm/G66U/7zWyEpF9L+pikXZLWSVro7lua2kgFZrZTUqe7lz4mbGZ/KekPku529/OzZf8qaZ+735z94Rzn7v/YIr3dJOkPZc/cnE0o0zFwZmlJV0i6WiUeu0RfC1TCcSvjzD9b0nZ33+HuhyXdL2l+CX20PHdfI2nfCYvnS1qZ3V6p/n88TVeht5bg7nvc/dns9n5Jx2eWLvXYJfoqRRnhnyTp5QH3d6m1pvx2ST8zs/VmtrjsZgYxMZs2XZL2SppYZjODqDpzczOdMLN0yxy7ema8zhsX/N5qjru/X9JlkpZkT29bkve/Zmul4ZqaZm5ulkFmlv6jMo9dvTNe562M8O+WdNaA+5OzZS3B3Xdnv3slPaTWm3245/gkqdnv3pL7+aNWmrl5sJml1QLHrpVmvC4j/OskTTezqWZ2qqQrJa0qoY+3MLP27EKMzKxd0ly13uzDqyQtym4vkvRIib28SavM3FxpZmmVfOxabsZrd2/6j6TL1X/F/0VJ/1RGDxX6eqek/85+Npfdm6T71P808Ij6r418QdLbJa2WtE3S45LGt1Bv90jaKOk59Qeto6Te5qj/Kf1zkjZkP5eXfewSfZVy3HiHHxAUF/yAoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwT1f91lbN87A+fQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcaad913438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Example of a picture\n",
    "train_data.shape\n",
    "index = 7\n",
    "plt.imshow(train_data[index].reshape(28, 28))\n",
    "print (\"y = \" + str(np.squeeze(train_labels[index])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of training examples = 55000\n",
      "number of evaluation examples = 10000\n",
      "X_train shape: (55000, 784)\n",
      "Y_train shape: (55000,)\n",
      "X_test shape: (10000, 784)\n",
      "Y_test shape: (10000,)\n"
     ]
    }
   ],
   "source": [
    "# get some statistic from the dataset\n",
    "print (\"number of training examples = \" + str(train_data.shape[0]))\n",
    "print (\"number of evaluation examples = \" + str(eval_data.shape[0]))\n",
    "print (\"X_train shape: \" + str(train_data.shape))\n",
    "print (\"Y_train shape: \" + str(train_labels.shape))\n",
    "print (\"X_test shape: \" + str(eval_data.shape))\n",
    "print (\"Y_test shape: \" + str(eval_labels.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see one of these 3x3 filters:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **4 - Implementing the full VGG-13 CNN network:**\n",
    "<br/>\n",
    "<img src=\"images/vgg.png\" style=\"width:1000px;height:150px;\">\n",
    "<br/>\n",
    "In TensorFlow, there are built-in functions that carry out the convolution steps.\n",
    "\n",
    "- **`tf.layers.conv2d(input, num_filters, filter_size, padding='same', activation=tf.nn.relu)`:** given an input and a group of filters, this function convolves filters on the input.\n",
    "\n",
    "- **`tf.layers.max_ppoling2d(input, pool_size, strides, padding='same')`:** given an input, this function uses a window of size `pool_size` and strides to carry out max pooling over each window.\n",
    "\n",
    "- **`tf.contrib.layers.flatten(P)`**: given an input P, this function flattens each example into a 1D vector it while maintaining the batch-size. It returns a flattened tensor with shape [batch_size, k].\n",
    "\n",
    "- **`tf.layers.dense(input, units, activation=tf.nn.relu)`:** given a the flattened input, it returns the output computed using a fully connected layer. The `units' determine the number of neuron in the FC layer.\n",
    "\n",
    "In the last function above (`tf.layers.dense`), the fully connected layer automatically initializes weights in the graph and keeps on training them as you train the model. Hence, we do not need to initialize those weights.\n",
    "\n",
    "Now we implement the `cnn_model_fn(features, labels, mode)` function below to build the following model: \n",
    "`2 x CONV2D -> RELU -> MAXPOOL -> 2 x CONV2D -> RELU -> MAXPOOL -> 2 x CONV2D -> RELU -> MAXPOOL -> 2 x CONV2D -> RELU -> MAXPOOL -> 2 x CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FC1 -> FC2 -> FC3 -> output`\n",
    "\n",
    "In detail, we will use the following parameters for all the steps:\n",
    "     - Conv2D: A 3 by 3 filter size with stride 1, padding is \"same\" with relu activation function\n",
    "     - Max pool: A 2 by 2 filter size with stride 2, padding is \"same\"\n",
    "     \n",
    "Here is the text version of the network in the original paper:\n",
    "<br/>\n",
    "<img src=\"images/vgg-text.png\" style=\"width:400px;height:400px;\">\n",
    "<br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_model_fn(features, labels, mode):\n",
    "    # Input Layer\n",
    "    input_height, input_width = 28, 28\n",
    "    input_channels = 1\n",
    "    input_layer = tf.reshape(features[\"x\"], [-1, input_height, input_width, input_channels])\n",
    "\n",
    "    # Convolutional Layer #1 and Pooling Layer #1\n",
    "    conv1_1 = tf.layers.conv2d(inputs=input_layer, filters=64, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    conv1_2 = tf.layers.conv2d(inputs=conv1_1, filters=64, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    pool1 = tf.layers.max_pooling2d(inputs=conv1_2, pool_size=[2, 2], strides=2, padding=\"same\")\n",
    "    \n",
    "    # Convolutional Layer #2 and Pooling Layer #2\n",
    "    conv2_1 = tf.layers.conv2d(inputs=pool1, filters=128, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    conv2_2 = tf.layers.conv2d(inputs=conv2_1, filters=128, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    pool2 = tf.layers.max_pooling2d(inputs=conv2_2, pool_size=[2, 2], strides=2, padding=\"same\")\n",
    "\n",
    "    # Convolutional Layer #3 and Pooling Layer #3\n",
    "    conv3_1 = tf.layers.conv2d(inputs=pool2, filters=256, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    conv3_2 = tf.layers.conv2d(inputs=conv3_1, filters=256, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    pool3 = tf.layers.max_pooling2d(inputs=conv3_2, pool_size=[2, 2], strides=2, padding=\"same\")\n",
    "\n",
    "    # Convolutional Layer #4 and Pooling Layer #4\n",
    "    conv4_1 = tf.layers.conv2d(inputs=pool3, filters=512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    conv4_2 = tf.layers.conv2d(inputs=conv4_1, filters=512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    pool4 = tf.layers.max_pooling2d(inputs=conv4_2, pool_size=[2, 2], strides=2, padding=\"same\")\n",
    "\n",
    "    # Convolutional Layer #5 and Pooling Layer #5\n",
    "    conv5_1 = tf.layers.conv2d(inputs=pool4, filters=512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    conv5_2 = tf.layers.conv2d(inputs=conv5_1, filters=512, kernel_size=[3, 3], padding=\"same\", activation=tf.nn.relu)\n",
    "    pool5 = tf.layers.max_pooling2d(inputs=conv5_2, pool_size=[2, 2], strides=2, padding=\"same\")\n",
    "\n",
    "    # FC Layers\n",
    "    pool5_flat = tf.contrib.layers.flatten(pool5)\n",
    "    FC1 = tf.layers.dense(inputs=pool5_flat, units=4096, activation=tf.nn.relu)\n",
    "    FC2 = tf.layers.dense(inputs=FC1, units=4096, activation=tf.nn.relu)\n",
    "    FC3 = tf.layers.dense(inputs=FC2, units=1000, activation=tf.nn.relu)\n",
    "\n",
    "    \"\"\"the training argument takes a boolean specifying whether or not the model is currently \n",
    "    being run in training mode; dropout will only be performed if training is true. here, \n",
    "    we check if the mode passed to our model function cnn_model_fn is train mode. \"\"\"\n",
    "    dropout = tf.layers.dropout(inputs=FC3, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    \n",
    "    # Logits Layer or the output layer. which will return the raw values for our predictions.\n",
    "    # Like FC layer, logits layer is another dense layer. We leave the activation function empty \n",
    "    # so we can apply the softmax\n",
    "    logits = tf.layers.dense(inputs=dropout, units=10)\n",
    "    \n",
    "    # Then we make predictions based on raw output\n",
    "    predictions = {\n",
    "        # Generate predictions (for PREDICT and EVAL mode)\n",
    "        # the predicted class for each example - a vlaue from 0-9\n",
    "        \"classes\": tf.argmax(input=logits, axis=1),\n",
    "        # to calculate the probablities for each target class we use the softmax\n",
    "        \"probabilities\": tf.nn.softmax(logits, name=\"softmax_tensor\")\n",
    "    }\n",
    "    \n",
    "    # so now our predictions are compiled in a dict object in python and using that we return an estimator object\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n",
    "    \n",
    "    \n",
    "    '''Calculate Loss (for both TRAIN and EVAL modes): computes the softmax entropy loss. \n",
    "    This function both computes the softmax activation function as well as the resulting loss.'''\n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "\n",
    "    # Configure the Training Options (for TRAIN mode)\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n",
    "        train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
    "\n",
    "    # Add evaluation metrics (for EVAL mode)\n",
    "    eval_metric_ops = {\n",
    "        \"accuracy\": tf.metrics.accuracy(labels=labels,\n",
    "                                        predictions=predictions[\"classes\"])}\n",
    "    return tf.estimator.EstimatorSpec(mode=mode,\n",
    "                                      loss=loss,\n",
    "                                      eval_metric_ops=eval_metric_ops)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_is_chief': True, '_tf_random_seed': None, '_task_id': 0, '_log_step_count_steps': 100, '_num_ps_replicas': 0, '_save_checkpoints_secs': 600, '_master': '', '_save_checkpoints_steps': None, '_task_type': 'worker', '_session_config': None, '_model_dir': '/tmp/mnist_vgg13_model', '_num_worker_replicas': 1, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f33766724a8>, '_keep_checkpoint_every_n_hours': 10000, '_keep_checkpoint_max': 5, '_save_summary_steps': 100, '_service': None}\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/mnist_vgg13_model/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.3024457, step = 1\n",
      "INFO:tensorflow:global_step/sec: 33.3624\n",
      "INFO:tensorflow:loss = 2.3024359, step = 101 (2.999 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.6312\n",
      "INFO:tensorflow:loss = 2.3019745, step = 201 (2.887 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5752\n",
      "INFO:tensorflow:loss = 2.3018668, step = 301 (2.893 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5009\n",
      "INFO:tensorflow:loss = 2.3017647, step = 401 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4899\n",
      "INFO:tensorflow:loss = 2.3017347, step = 501 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5969\n",
      "INFO:tensorflow:loss = 2.302085, step = 601 (2.890 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5601\n",
      "INFO:tensorflow:loss = 2.3024077, step = 701 (2.893 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.622\n",
      "INFO:tensorflow:loss = 2.301997, step = 801 (2.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5256\n",
      "INFO:tensorflow:loss = 2.2995162, step = 901 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5693\n",
      "INFO:tensorflow:loss = 2.3017197, step = 1001 (2.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4813\n",
      "INFO:tensorflow:loss = 2.3020413, step = 1101 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5225\n",
      "INFO:tensorflow:loss = 2.3016045, step = 1201 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5461\n",
      "INFO:tensorflow:loss = 2.299843, step = 1301 (2.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.6092\n",
      "INFO:tensorflow:loss = 2.302646, step = 1401 (2.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4563\n",
      "INFO:tensorflow:loss = 2.3046136, step = 1501 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.6107\n",
      "INFO:tensorflow:loss = 2.3008747, step = 1601 (2.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.6494\n",
      "INFO:tensorflow:loss = 2.3018627, step = 1701 (2.886 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4878\n",
      "INFO:tensorflow:loss = 2.303177, step = 1801 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5662\n",
      "INFO:tensorflow:loss = 2.3010514, step = 1901 (2.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5638\n",
      "INFO:tensorflow:loss = 2.3012455, step = 2001 (2.892 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4407\n",
      "INFO:tensorflow:loss = 2.2990637, step = 2101 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.509\n",
      "INFO:tensorflow:loss = 2.2992778, step = 2201 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4529\n",
      "INFO:tensorflow:loss = 2.3018408, step = 2301 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.44\n",
      "INFO:tensorflow:loss = 2.298725, step = 2401 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5301\n",
      "INFO:tensorflow:loss = 2.3020263, step = 2501 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3737\n",
      "INFO:tensorflow:loss = 2.3010063, step = 2601 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4515\n",
      "INFO:tensorflow:loss = 2.2988348, step = 2701 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4925\n",
      "INFO:tensorflow:loss = 2.3021977, step = 2801 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4358\n",
      "INFO:tensorflow:loss = 2.3001924, step = 2901 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4623\n",
      "INFO:tensorflow:loss = 2.3055842, step = 3001 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5151\n",
      "INFO:tensorflow:loss = 2.2977085, step = 3101 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5843\n",
      "INFO:tensorflow:loss = 2.3002486, step = 3201 (2.891 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4508\n",
      "INFO:tensorflow:loss = 2.3020186, step = 3301 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.6429\n",
      "INFO:tensorflow:loss = 2.2999961, step = 3401 (2.887 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5238\n",
      "INFO:tensorflow:loss = 2.3028052, step = 3501 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.421\n",
      "INFO:tensorflow:loss = 2.304041, step = 3601 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5121\n",
      "INFO:tensorflow:loss = 2.294374, step = 3701 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4116\n",
      "INFO:tensorflow:loss = 2.300902, step = 3801 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.535\n",
      "INFO:tensorflow:loss = 2.2969353, step = 3901 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4188\n",
      "INFO:tensorflow:loss = 2.295891, step = 4001 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4829\n",
      "INFO:tensorflow:loss = 2.300285, step = 4101 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4853\n",
      "INFO:tensorflow:loss = 2.2945442, step = 4201 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4991\n",
      "INFO:tensorflow:loss = 2.298645, step = 4301 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.419\n",
      "INFO:tensorflow:loss = 2.298185, step = 4401 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3652\n",
      "INFO:tensorflow:loss = 2.3001742, step = 4501 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4889\n",
      "INFO:tensorflow:loss = 2.3088653, step = 4601 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3299\n",
      "INFO:tensorflow:loss = 2.300877, step = 4701 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4932\n",
      "INFO:tensorflow:loss = 2.300152, step = 4801 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4951\n",
      "INFO:tensorflow:loss = 2.296337, step = 4901 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4819\n",
      "INFO:tensorflow:loss = 2.2955093, step = 5001 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4813\n",
      "INFO:tensorflow:loss = 2.304335, step = 5101 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5706\n",
      "INFO:tensorflow:loss = 2.2947235, step = 5201 (2.892 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3982\n",
      "INFO:tensorflow:loss = 2.2973878, step = 5301 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4932\n",
      "INFO:tensorflow:loss = 2.3008888, step = 5401 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5364\n",
      "INFO:tensorflow:loss = 2.3009605, step = 5501 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5144\n",
      "INFO:tensorflow:loss = 2.2986796, step = 5601 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4717\n",
      "INFO:tensorflow:loss = 2.303681, step = 5701 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4767\n",
      "INFO:tensorflow:loss = 2.2990258, step = 5801 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4966\n",
      "INFO:tensorflow:loss = 2.3043134, step = 5901 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3172\n",
      "INFO:tensorflow:loss = 2.2913117, step = 6001 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4504\n",
      "INFO:tensorflow:loss = 2.3046463, step = 6101 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4659\n",
      "INFO:tensorflow:loss = 2.3063188, step = 6201 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4961\n",
      "INFO:tensorflow:loss = 2.2940476, step = 6301 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4422\n",
      "INFO:tensorflow:loss = 2.2874484, step = 6401 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2053\n",
      "INFO:tensorflow:loss = 2.293964, step = 6501 (2.923 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3393\n",
      "INFO:tensorflow:loss = 2.2963052, step = 6601 (2.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.7936\n",
      "INFO:tensorflow:loss = 2.2963097, step = 6701 (2.959 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.7944\n",
      "INFO:tensorflow:loss = 2.3005474, step = 6801 (2.959 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3438\n",
      "INFO:tensorflow:loss = 2.3006144, step = 6901 (2.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4232\n",
      "INFO:tensorflow:loss = 2.3014119, step = 7001 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3927\n",
      "INFO:tensorflow:loss = 2.3038461, step = 7101 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5292\n",
      "INFO:tensorflow:loss = 2.3030076, step = 7201 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4602\n",
      "INFO:tensorflow:loss = 2.3008718, step = 7301 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4297\n",
      "INFO:tensorflow:loss = 2.3013227, step = 7401 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5601\n",
      "INFO:tensorflow:loss = 2.301737, step = 7501 (2.893 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4562\n",
      "INFO:tensorflow:loss = 2.2970622, step = 7601 (2.903 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 34.4841\n",
      "INFO:tensorflow:loss = 2.2955787, step = 7701 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.475\n",
      "INFO:tensorflow:loss = 2.3022883, step = 7801 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4965\n",
      "INFO:tensorflow:loss = 2.2989097, step = 7901 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4934\n",
      "INFO:tensorflow:loss = 2.2917895, step = 8001 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4095\n",
      "INFO:tensorflow:loss = 2.3022003, step = 8101 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4383\n",
      "INFO:tensorflow:loss = 2.3011668, step = 8201 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5911\n",
      "INFO:tensorflow:loss = 2.2943037, step = 8301 (2.891 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4508\n",
      "INFO:tensorflow:loss = 2.289969, step = 8401 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4869\n",
      "INFO:tensorflow:loss = 2.305867, step = 8501 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4053\n",
      "INFO:tensorflow:loss = 2.294767, step = 8601 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3497\n",
      "INFO:tensorflow:loss = 2.2965217, step = 8701 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4707\n",
      "INFO:tensorflow:loss = 2.3024015, step = 8801 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.36\n",
      "INFO:tensorflow:loss = 2.2935913, step = 8901 (2.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5192\n",
      "INFO:tensorflow:loss = 2.2928903, step = 9001 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5088\n",
      "INFO:tensorflow:loss = 2.290739, step = 9101 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.531\n",
      "INFO:tensorflow:loss = 2.2919953, step = 9201 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5079\n",
      "INFO:tensorflow:loss = 2.3054361, step = 9301 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3385\n",
      "INFO:tensorflow:loss = 2.3006704, step = 9401 (2.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4101\n",
      "INFO:tensorflow:loss = 2.2935412, step = 9501 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3681\n",
      "INFO:tensorflow:loss = 2.304914, step = 9601 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5252\n",
      "INFO:tensorflow:loss = 2.2944622, step = 9701 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5699\n",
      "INFO:tensorflow:loss = 2.2922823, step = 9801 (2.892 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4699\n",
      "INFO:tensorflow:loss = 2.2917752, step = 9901 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4106\n",
      "INFO:tensorflow:loss = 2.2956376, step = 10001 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4543\n",
      "INFO:tensorflow:loss = 2.308194, step = 10101 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4553\n",
      "INFO:tensorflow:loss = 2.2918105, step = 10201 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5069\n",
      "INFO:tensorflow:loss = 2.3023002, step = 10301 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4869\n",
      "INFO:tensorflow:loss = 2.2915814, step = 10401 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5196\n",
      "INFO:tensorflow:loss = 2.2899761, step = 10501 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4738\n",
      "INFO:tensorflow:loss = 2.2913756, step = 10601 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4489\n",
      "INFO:tensorflow:loss = 2.299275, step = 10701 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4016\n",
      "INFO:tensorflow:loss = 2.2974885, step = 10801 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4573\n",
      "INFO:tensorflow:loss = 2.2968884, step = 10901 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4934\n",
      "INFO:tensorflow:loss = 2.2964563, step = 11001 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4604\n",
      "INFO:tensorflow:loss = 2.297178, step = 11101 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.414\n",
      "INFO:tensorflow:loss = 2.3074737, step = 11201 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4784\n",
      "INFO:tensorflow:loss = 2.2964869, step = 11301 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4417\n",
      "INFO:tensorflow:loss = 2.288181, step = 11401 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3575\n",
      "INFO:tensorflow:loss = 2.291268, step = 11501 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4178\n",
      "INFO:tensorflow:loss = 2.296508, step = 11601 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4744\n",
      "INFO:tensorflow:loss = 2.3006089, step = 11701 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5059\n",
      "INFO:tensorflow:loss = 2.299033, step = 11801 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5237\n",
      "INFO:tensorflow:loss = 2.297297, step = 11901 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5339\n",
      "INFO:tensorflow:loss = 2.2914016, step = 12001 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3775\n",
      "INFO:tensorflow:loss = 2.2950516, step = 12101 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4871\n",
      "INFO:tensorflow:loss = 2.296091, step = 12201 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5095\n",
      "INFO:tensorflow:loss = 2.299985, step = 12301 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4694\n",
      "INFO:tensorflow:loss = 2.2965088, step = 12401 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4535\n",
      "INFO:tensorflow:loss = 2.3056197, step = 12501 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5038\n",
      "INFO:tensorflow:loss = 2.2931943, step = 12601 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5521\n",
      "INFO:tensorflow:loss = 2.3019264, step = 12701 (2.895 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.524\n",
      "INFO:tensorflow:loss = 2.2957299, step = 12801 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4194\n",
      "INFO:tensorflow:loss = 2.2915597, step = 12901 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5103\n",
      "INFO:tensorflow:loss = 2.2979085, step = 13001 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3933\n",
      "INFO:tensorflow:loss = 2.296691, step = 13101 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.548\n",
      "INFO:tensorflow:loss = 2.2956371, step = 13201 (2.894 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2938\n",
      "INFO:tensorflow:loss = 2.2996137, step = 13301 (2.916 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4562\n",
      "INFO:tensorflow:loss = 2.2955573, step = 13401 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4908\n",
      "INFO:tensorflow:loss = 2.2917125, step = 13501 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4662\n",
      "INFO:tensorflow:loss = 2.2897487, step = 13601 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4675\n",
      "INFO:tensorflow:loss = 2.2880452, step = 13701 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4157\n",
      "INFO:tensorflow:loss = 2.2984142, step = 13801 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.538\n",
      "INFO:tensorflow:loss = 2.291024, step = 13901 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5012\n",
      "INFO:tensorflow:loss = 2.297457, step = 14001 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5134\n",
      "INFO:tensorflow:loss = 2.2918186, step = 14101 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4528\n",
      "INFO:tensorflow:loss = 2.2819006, step = 14201 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4059\n",
      "INFO:tensorflow:loss = 2.2788, step = 14301 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4766\n",
      "INFO:tensorflow:loss = 2.2878735, step = 14401 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4947\n",
      "INFO:tensorflow:loss = 2.2849054, step = 14501 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.537\n",
      "INFO:tensorflow:loss = 2.2909455, step = 14601 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4175\n",
      "INFO:tensorflow:loss = 2.2895265, step = 14701 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4696\n",
      "INFO:tensorflow:loss = 2.292672, step = 14801 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3989\n",
      "INFO:tensorflow:loss = 2.289581, step = 14901 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4683\n",
      "INFO:tensorflow:loss = 2.3011446, step = 15001 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4893\n",
      "INFO:tensorflow:loss = 2.2783515, step = 15101 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4797\n",
      "INFO:tensorflow:loss = 2.2943468, step = 15201 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4295\n",
      "INFO:tensorflow:loss = 2.2830207, step = 15301 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.586\n",
      "INFO:tensorflow:loss = 2.282056, step = 15401 (2.891 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5078\n",
      "INFO:tensorflow:loss = 2.2915397, step = 15501 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3727\n",
      "INFO:tensorflow:loss = 2.2803235, step = 15601 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.455\n",
      "INFO:tensorflow:loss = 2.2858138, step = 15701 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4033\n",
      "INFO:tensorflow:loss = 2.2827718, step = 15801 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5073\n",
      "INFO:tensorflow:loss = 2.2796848, step = 15901 (2.898 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 34.6123\n",
      "INFO:tensorflow:loss = 2.2803714, step = 16001 (2.889 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3353\n",
      "INFO:tensorflow:loss = 2.2763321, step = 16101 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4104\n",
      "INFO:tensorflow:loss = 2.2728038, step = 16201 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5362\n",
      "INFO:tensorflow:loss = 2.2822785, step = 16301 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5003\n",
      "INFO:tensorflow:loss = 2.2781374, step = 16401 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5308\n",
      "INFO:tensorflow:loss = 2.2940154, step = 16501 (2.896 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.514\n",
      "INFO:tensorflow:loss = 2.273203, step = 16601 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3568\n",
      "INFO:tensorflow:loss = 2.280427, step = 16701 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4099\n",
      "INFO:tensorflow:loss = 2.269772, step = 16801 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4496\n",
      "INFO:tensorflow:loss = 2.2912347, step = 16901 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3139\n",
      "INFO:tensorflow:loss = 2.2758846, step = 17001 (2.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4663\n",
      "INFO:tensorflow:loss = 2.2791293, step = 17101 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4483\n",
      "INFO:tensorflow:loss = 2.2744672, step = 17201 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2255\n",
      "INFO:tensorflow:loss = 2.2597907, step = 17301 (2.923 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3195\n",
      "INFO:tensorflow:loss = 2.2481544, step = 17401 (2.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3739\n",
      "INFO:tensorflow:loss = 2.254632, step = 17501 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5638\n",
      "INFO:tensorflow:loss = 2.2638025, step = 17601 (2.892 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.344\n",
      "INFO:tensorflow:loss = 2.2652748, step = 17701 (2.751 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2373\n",
      "INFO:tensorflow:loss = 2.2512665, step = 17801 (2.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2043\n",
      "INFO:tensorflow:loss = 2.2461, step = 17901 (2.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2368\n",
      "INFO:tensorflow:loss = 2.2644806, step = 18001 (2.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.265\n",
      "INFO:tensorflow:loss = 2.2426455, step = 18101 (2.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.3136\n",
      "INFO:tensorflow:loss = 2.2419274, step = 18201 (2.754 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.3454\n",
      "INFO:tensorflow:loss = 2.2160437, step = 18301 (2.751 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2875\n",
      "INFO:tensorflow:loss = 2.2264907, step = 18401 (2.756 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1471\n",
      "INFO:tensorflow:loss = 2.20353, step = 18501 (2.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2534\n",
      "INFO:tensorflow:loss = 2.1761198, step = 18601 (2.758 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.128\n",
      "INFO:tensorflow:loss = 2.2142687, step = 18701 (2.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0003\n",
      "INFO:tensorflow:loss = 2.1689043, step = 18801 (2.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1979\n",
      "INFO:tensorflow:loss = 2.1174443, step = 18901 (2.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1655\n",
      "INFO:tensorflow:loss = 2.1454933, step = 19001 (2.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9877\n",
      "INFO:tensorflow:loss = 2.0731287, step = 19101 (2.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1454\n",
      "INFO:tensorflow:loss = 2.033677, step = 19201 (2.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2363\n",
      "INFO:tensorflow:loss = 1.877105, step = 19301 (2.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1283\n",
      "INFO:tensorflow:loss = 1.7496741, step = 19401 (2.768 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2211\n",
      "INFO:tensorflow:loss = 1.5991629, step = 19501 (2.761 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2022\n",
      "INFO:tensorflow:loss = 1.4544545, step = 19601 (2.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1182\n",
      "INFO:tensorflow:loss = 1.2201507, step = 19701 (2.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0398\n",
      "INFO:tensorflow:loss = 1.1744809, step = 19801 (2.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.3128\n",
      "INFO:tensorflow:loss = 1.0420521, step = 19901 (2.754 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0255\n",
      "INFO:tensorflow:loss = 0.91135335, step = 20001 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.0344\n",
      "INFO:tensorflow:loss = 0.7763748, step = 20101 (2.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2921\n",
      "INFO:tensorflow:loss = 0.84598696, step = 20201 (2.916 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.7699\n",
      "INFO:tensorflow:loss = 1.104431, step = 20301 (2.962 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0569\n",
      "INFO:tensorflow:loss = 0.64550763, step = 20401 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3234\n",
      "INFO:tensorflow:loss = 0.7454869, step = 20501 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4078\n",
      "INFO:tensorflow:loss = 0.8152907, step = 20601 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3586\n",
      "INFO:tensorflow:loss = 0.61944234, step = 20701 (2.912 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20773 into /tmp/mnist_vgg13_model/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 30.8287\n",
      "INFO:tensorflow:loss = 0.62630266, step = 20801 (3.242 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2991\n",
      "INFO:tensorflow:loss = 0.66512865, step = 20901 (2.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3867\n",
      "INFO:tensorflow:loss = 0.53541744, step = 21001 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3647\n",
      "INFO:tensorflow:loss = 0.43956357, step = 21101 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3201\n",
      "INFO:tensorflow:loss = 0.48911607, step = 21201 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4234\n",
      "INFO:tensorflow:loss = 0.54616576, step = 21301 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2114\n",
      "INFO:tensorflow:loss = 0.42734948, step = 21401 (2.923 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3259\n",
      "INFO:tensorflow:loss = 0.4610698, step = 21501 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3823\n",
      "INFO:tensorflow:loss = 0.7562766, step = 21601 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.423\n",
      "INFO:tensorflow:loss = 0.41100252, step = 21701 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2772\n",
      "INFO:tensorflow:loss = 0.44463578, step = 21801 (2.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.311\n",
      "INFO:tensorflow:loss = 0.3469187, step = 21901 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3204\n",
      "INFO:tensorflow:loss = 0.29334316, step = 22001 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4372\n",
      "INFO:tensorflow:loss = 0.4306474, step = 22101 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4386\n",
      "INFO:tensorflow:loss = 0.3251673, step = 22201 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3233\n",
      "INFO:tensorflow:loss = 0.32676265, step = 22301 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3865\n",
      "INFO:tensorflow:loss = 0.23600054, step = 22401 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3942\n",
      "INFO:tensorflow:loss = 0.301925, step = 22501 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3143\n",
      "INFO:tensorflow:loss = 0.39321518, step = 22601 (2.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3103\n",
      "INFO:tensorflow:loss = 0.28859532, step = 22701 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3674\n",
      "INFO:tensorflow:loss = 0.1502078, step = 22801 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3196\n",
      "INFO:tensorflow:loss = 0.35744792, step = 22901 (2.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3179\n",
      "INFO:tensorflow:loss = 0.35357732, step = 23001 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3662\n",
      "INFO:tensorflow:loss = 0.2364287, step = 23101 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3295\n",
      "INFO:tensorflow:loss = 0.31806374, step = 23201 (2.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2276\n",
      "INFO:tensorflow:loss = 0.2648021, step = 23301 (2.922 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3907\n",
      "INFO:tensorflow:loss = 0.32277462, step = 23401 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.418\n",
      "INFO:tensorflow:loss = 0.17560521, step = 23501 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2274\n",
      "INFO:tensorflow:loss = 0.1768446, step = 23601 (2.922 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.221\n",
      "INFO:tensorflow:loss = 0.32288405, step = 23701 (2.922 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1239\n",
      "INFO:tensorflow:loss = 0.36192417, step = 23801 (2.931 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3331\n",
      "INFO:tensorflow:loss = 0.21200134, step = 23901 (2.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3964\n",
      "INFO:tensorflow:loss = 0.20915276, step = 24001 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5004\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.24431635, step = 24101 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3761\n",
      "INFO:tensorflow:loss = 0.15338968, step = 24201 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5041\n",
      "INFO:tensorflow:loss = 0.14086466, step = 24301 (2.899 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3625\n",
      "INFO:tensorflow:loss = 0.3405792, step = 24401 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3645\n",
      "INFO:tensorflow:loss = 0.30023575, step = 24501 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.453\n",
      "INFO:tensorflow:loss = 0.16768338, step = 24601 (2.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3708\n",
      "INFO:tensorflow:loss = 0.23954768, step = 24701 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4507\n",
      "INFO:tensorflow:loss = 0.23914015, step = 24801 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.141\n",
      "INFO:tensorflow:loss = 0.4081696, step = 24901 (2.928 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3163\n",
      "INFO:tensorflow:loss = 0.17421135, step = 25001 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2876\n",
      "INFO:tensorflow:loss = 0.25084838, step = 25101 (2.916 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3972\n",
      "INFO:tensorflow:loss = 0.10671947, step = 25201 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3963\n",
      "INFO:tensorflow:loss = 0.13161904, step = 25301 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3655\n",
      "INFO:tensorflow:loss = 0.18540622, step = 25401 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1951\n",
      "INFO:tensorflow:loss = 0.12228361, step = 25501 (2.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.35\n",
      "INFO:tensorflow:loss = 0.2515322, step = 25601 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5142\n",
      "INFO:tensorflow:loss = 0.1410765, step = 25701 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3377\n",
      "INFO:tensorflow:loss = 0.29322743, step = 25801 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4415\n",
      "INFO:tensorflow:loss = 0.19938941, step = 25901 (2.903 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2672\n",
      "INFO:tensorflow:loss = 0.21614161, step = 26001 (2.919 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3818\n",
      "INFO:tensorflow:loss = 0.23770542, step = 26101 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4758\n",
      "INFO:tensorflow:loss = 0.302389, step = 26201 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3579\n",
      "INFO:tensorflow:loss = 0.1164901, step = 26301 (2.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3011\n",
      "INFO:tensorflow:loss = 0.2298138, step = 26401 (2.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3253\n",
      "INFO:tensorflow:loss = 0.21029325, step = 26501 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3734\n",
      "INFO:tensorflow:loss = 0.14498043, step = 26601 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2637\n",
      "INFO:tensorflow:loss = 0.08261353, step = 26701 (2.918 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4106\n",
      "INFO:tensorflow:loss = 0.09480828, step = 26801 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2897\n",
      "INFO:tensorflow:loss = 0.089494646, step = 26901 (2.917 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3321\n",
      "INFO:tensorflow:loss = 0.21164696, step = 27001 (2.913 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3365\n",
      "INFO:tensorflow:loss = 0.18242997, step = 27101 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3928\n",
      "INFO:tensorflow:loss = 0.23555318, step = 27201 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1963\n",
      "INFO:tensorflow:loss = 0.098083384, step = 27301 (2.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3147\n",
      "INFO:tensorflow:loss = 0.10893319, step = 27401 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.18\n",
      "INFO:tensorflow:loss = 0.12583004, step = 27501 (2.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.252\n",
      "INFO:tensorflow:loss = 0.19990687, step = 27601 (2.919 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4114\n",
      "INFO:tensorflow:loss = 0.22673579, step = 27701 (2.906 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3011\n",
      "INFO:tensorflow:loss = 0.12745324, step = 27801 (2.915 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3657\n",
      "INFO:tensorflow:loss = 0.053561073, step = 27901 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5166\n",
      "INFO:tensorflow:loss = 0.18809101, step = 28001 (2.897 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4271\n",
      "INFO:tensorflow:loss = 0.24462126, step = 28101 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4576\n",
      "INFO:tensorflow:loss = 0.21433726, step = 28201 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.425\n",
      "INFO:tensorflow:loss = 0.093781024, step = 28301 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4063\n",
      "INFO:tensorflow:loss = 0.20117229, step = 28401 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.483\n",
      "INFO:tensorflow:loss = 0.06789169, step = 28501 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4406\n",
      "INFO:tensorflow:loss = 0.15778366, step = 28601 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.367\n",
      "INFO:tensorflow:loss = 0.089592345, step = 28701 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3201\n",
      "INFO:tensorflow:loss = 0.124576695, step = 28801 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.398\n",
      "INFO:tensorflow:loss = 0.20963077, step = 28901 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3538\n",
      "INFO:tensorflow:loss = 0.12837438, step = 29001 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3514\n",
      "INFO:tensorflow:loss = 0.19518423, step = 29101 (2.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4568\n",
      "INFO:tensorflow:loss = 0.09408713, step = 29201 (2.902 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3403\n",
      "INFO:tensorflow:loss = 0.11151289, step = 29301 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2021\n",
      "INFO:tensorflow:loss = 0.22152203, step = 29401 (2.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3148\n",
      "INFO:tensorflow:loss = 0.19423914, step = 29501 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.5091\n",
      "INFO:tensorflow:loss = 0.06485153, step = 29601 (2.898 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3775\n",
      "INFO:tensorflow:loss = 0.06326921, step = 29701 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4783\n",
      "INFO:tensorflow:loss = 0.059113454, step = 29801 (2.900 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1899\n",
      "INFO:tensorflow:loss = 0.08544799, step = 29901 (2.925 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3568\n",
      "INFO:tensorflow:loss = 0.18356805, step = 30001 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3914\n",
      "INFO:tensorflow:loss = 0.024621045, step = 30101 (2.907 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3715\n",
      "INFO:tensorflow:loss = 0.10343031, step = 30201 (2.909 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3641\n",
      "INFO:tensorflow:loss = 0.10378599, step = 30301 (2.910 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2082\n",
      "INFO:tensorflow:loss = 0.08055232, step = 30401 (2.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3504\n",
      "INFO:tensorflow:loss = 0.04447268, step = 30501 (2.911 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.4125\n",
      "INFO:tensorflow:loss = 0.1711489, step = 30601 (2.905 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3405\n",
      "INFO:tensorflow:loss = 0.1260111, step = 30701 (2.912 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3779\n",
      "INFO:tensorflow:loss = 0.060509883, step = 30801 (2.908 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.3107\n",
      "INFO:tensorflow:loss = 0.0706469, step = 30901 (2.914 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.439\n",
      "INFO:tensorflow:loss = 0.23554975, step = 31001 (2.904 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.4875\n",
      "INFO:tensorflow:loss = 0.09704328, step = 31101 (2.987 sec)\n",
      "INFO:tensorflow:global_step/sec: 32.9613\n",
      "INFO:tensorflow:loss = 0.08640723, step = 31201 (3.045 sec)\n",
      "INFO:tensorflow:global_step/sec: 32.4999\n",
      "INFO:tensorflow:loss = 0.08623208, step = 31301 (3.067 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.524\n",
      "INFO:tensorflow:loss = 0.11800768, step = 31401 (2.980 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2797\n",
      "INFO:tensorflow:loss = 0.19195801, step = 31501 (2.756 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0365\n",
      "INFO:tensorflow:loss = 0.12876987, step = 31601 (2.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9924\n",
      "INFO:tensorflow:loss = 0.15105191, step = 31701 (2.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.275\n",
      "INFO:tensorflow:loss = 0.045690965, step = 31801 (2.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1003\n",
      "INFO:tensorflow:loss = 0.0902389, step = 31901 (2.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0741\n",
      "INFO:tensorflow:loss = 0.109735385, step = 32001 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2143\n",
      "INFO:tensorflow:loss = 0.060215015, step = 32101 (2.761 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0738\n",
      "INFO:tensorflow:loss = 0.09614294, step = 32201 (2.772 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 36.0192\n",
      "INFO:tensorflow:loss = 0.08814603, step = 32301 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1936\n",
      "INFO:tensorflow:loss = 0.0915273, step = 32401 (2.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9919\n",
      "INFO:tensorflow:loss = 0.1559304, step = 32501 (2.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0224\n",
      "INFO:tensorflow:loss = 0.13064772, step = 32601 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2077\n",
      "INFO:tensorflow:loss = 0.186578, step = 32701 (2.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0741\n",
      "INFO:tensorflow:loss = 0.123838395, step = 32801 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9094\n",
      "INFO:tensorflow:loss = 0.055431586, step = 32901 (2.785 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1082\n",
      "INFO:tensorflow:loss = 0.08604826, step = 33001 (2.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1172\n",
      "INFO:tensorflow:loss = 0.16571075, step = 33101 (2.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.025\n",
      "INFO:tensorflow:loss = 0.174734, step = 33201 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1973\n",
      "INFO:tensorflow:loss = 0.13930923, step = 33301 (2.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2293\n",
      "INFO:tensorflow:loss = 0.069082886, step = 33401 (2.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.8319\n",
      "INFO:tensorflow:loss = 0.09529216, step = 33501 (2.791 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0321\n",
      "INFO:tensorflow:loss = 0.1427801, step = 33601 (2.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.98\n",
      "INFO:tensorflow:loss = 0.14456731, step = 33701 (2.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.8531\n",
      "INFO:tensorflow:loss = 0.06332739, step = 33801 (2.789 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0827\n",
      "INFO:tensorflow:loss = 0.060871318, step = 33901 (2.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2873\n",
      "INFO:tensorflow:loss = 0.043597627, step = 34001 (2.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9503\n",
      "INFO:tensorflow:loss = 0.06823686, step = 34101 (2.780 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0235\n",
      "INFO:tensorflow:loss = 0.0858595, step = 34201 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1049\n",
      "INFO:tensorflow:loss = 0.026161574, step = 34301 (2.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0796\n",
      "INFO:tensorflow:loss = 0.22241515, step = 34401 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9463\n",
      "INFO:tensorflow:loss = 0.075593024, step = 34501 (2.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1353\n",
      "INFO:tensorflow:loss = 0.056470156, step = 34601 (2.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0623\n",
      "INFO:tensorflow:loss = 0.06877685, step = 34701 (2.773 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.157\n",
      "INFO:tensorflow:loss = 0.09656765, step = 34801 (2.766 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2475\n",
      "INFO:tensorflow:loss = 0.099831074, step = 34901 (2.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.961\n",
      "INFO:tensorflow:loss = 0.09322438, step = 35001 (2.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0245\n",
      "INFO:tensorflow:loss = 0.064449765, step = 35101 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.3573\n",
      "INFO:tensorflow:loss = 0.076590165, step = 35201 (2.751 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0065\n",
      "INFO:tensorflow:loss = 0.03158188, step = 35301 (2.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.098\n",
      "INFO:tensorflow:loss = 0.073423944, step = 35401 (2.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2744\n",
      "INFO:tensorflow:loss = 0.12235273, step = 35501 (2.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0906\n",
      "INFO:tensorflow:loss = 0.035940044, step = 35601 (2.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.966\n",
      "INFO:tensorflow:loss = 0.01997225, step = 35701 (2.780 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.989\n",
      "INFO:tensorflow:loss = 0.058835253, step = 35801 (2.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.7869\n",
      "INFO:tensorflow:loss = 0.26647836, step = 35901 (2.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0007\n",
      "INFO:tensorflow:loss = 0.08537402, step = 36001 (2.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.272\n",
      "INFO:tensorflow:loss = 0.08744536, step = 36101 (2.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1163\n",
      "INFO:tensorflow:loss = 0.14321242, step = 36201 (2.769 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0211\n",
      "INFO:tensorflow:loss = 0.098565996, step = 36301 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1422\n",
      "INFO:tensorflow:loss = 0.123108834, step = 36401 (2.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0368\n",
      "INFO:tensorflow:loss = 0.14363009, step = 36501 (2.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9592\n",
      "INFO:tensorflow:loss = 0.08085214, step = 36601 (2.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2002\n",
      "INFO:tensorflow:loss = 0.048654962, step = 36701 (2.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9429\n",
      "INFO:tensorflow:loss = 0.12883109, step = 36801 (2.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9977\n",
      "INFO:tensorflow:loss = 0.10874418, step = 36901 (2.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2083\n",
      "INFO:tensorflow:loss = 0.14202826, step = 37001 (2.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0924\n",
      "INFO:tensorflow:loss = 0.017829752, step = 37101 (2.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9699\n",
      "INFO:tensorflow:loss = 0.089890435, step = 37201 (2.780 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2181\n",
      "INFO:tensorflow:loss = 0.101865895, step = 37301 (2.761 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0822\n",
      "INFO:tensorflow:loss = 0.06517927, step = 37401 (2.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0823\n",
      "INFO:tensorflow:loss = 0.045720957, step = 37501 (2.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.092\n",
      "INFO:tensorflow:loss = 0.09515615, step = 37601 (2.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9884\n",
      "INFO:tensorflow:loss = 0.087062925, step = 37701 (2.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0161\n",
      "INFO:tensorflow:loss = 0.05821681, step = 37801 (2.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0712\n",
      "INFO:tensorflow:loss = 0.061003666, step = 37901 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.049\n",
      "INFO:tensorflow:loss = 0.07524961, step = 38001 (2.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9369\n",
      "INFO:tensorflow:loss = 0.044006184, step = 38101 (2.783 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0923\n",
      "INFO:tensorflow:loss = 0.12144165, step = 38201 (2.771 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1654\n",
      "INFO:tensorflow:loss = 0.04819357, step = 38301 (2.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0369\n",
      "INFO:tensorflow:loss = 0.022777801, step = 38401 (2.775 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2295\n",
      "INFO:tensorflow:loss = 0.14948855, step = 38501 (2.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1887\n",
      "INFO:tensorflow:loss = 0.13120806, step = 38601 (2.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.8765\n",
      "INFO:tensorflow:loss = 0.033269357, step = 38701 (2.787 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.231\n",
      "INFO:tensorflow:loss = 0.19057533, step = 38801 (2.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1377\n",
      "INFO:tensorflow:loss = 0.06238319, step = 38901 (2.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.005\n",
      "INFO:tensorflow:loss = 0.060452543, step = 39001 (2.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1673\n",
      "INFO:tensorflow:loss = 0.03292608, step = 39101 (2.765 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2532\n",
      "INFO:tensorflow:loss = 0.055235248, step = 39201 (2.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0936\n",
      "INFO:tensorflow:loss = 0.17177859, step = 39301 (2.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.996\n",
      "INFO:tensorflow:loss = 0.071421154, step = 39401 (2.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1925\n",
      "INFO:tensorflow:loss = 0.08682945, step = 39501 (2.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0053\n",
      "INFO:tensorflow:loss = 0.1141587, step = 39601 (2.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0183\n",
      "INFO:tensorflow:loss = 0.042499423, step = 39701 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2112\n",
      "INFO:tensorflow:loss = 0.061068784, step = 39801 (2.762 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.058\n",
      "INFO:tensorflow:loss = 0.085514374, step = 39901 (2.773 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.043\n",
      "INFO:tensorflow:loss = 0.12619796, step = 40001 (2.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2798\n",
      "INFO:tensorflow:loss = 0.027635837, step = 40101 (2.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9769\n",
      "INFO:tensorflow:loss = 0.055071775, step = 40201 (2.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.948\n",
      "INFO:tensorflow:loss = 0.022936821, step = 40301 (2.782 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1747\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.018869003, step = 40401 (2.764 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.9838\n",
      "INFO:tensorflow:loss = 0.041131902, step = 40501 (2.779 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0792\n",
      "INFO:tensorflow:loss = 0.038472675, step = 40601 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2724\n",
      "INFO:tensorflow:loss = 0.09293892, step = 40701 (2.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0643\n",
      "INFO:tensorflow:loss = 0.03692863, step = 40801 (2.773 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1034\n",
      "INFO:tensorflow:loss = 0.057929385, step = 40901 (2.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2877\n",
      "INFO:tensorflow:loss = 0.039613787, step = 41001 (2.756 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0072\n",
      "INFO:tensorflow:loss = 0.05707495, step = 41101 (2.777 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.077\n",
      "INFO:tensorflow:loss = 0.08689953, step = 41201 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1801\n",
      "INFO:tensorflow:loss = 0.070070446, step = 41301 (2.764 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0826\n",
      "INFO:tensorflow:loss = 0.07234741, step = 41401 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0209\n",
      "INFO:tensorflow:loss = 0.009989007, step = 41501 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.3055\n",
      "INFO:tensorflow:loss = 0.03614047, step = 41601 (2.754 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0969\n",
      "INFO:tensorflow:loss = 0.081055775, step = 41701 (2.770 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0585\n",
      "INFO:tensorflow:loss = 0.05276156, step = 41801 (2.773 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 41861 into /tmp/mnist_vgg13_model/model.ckpt.\n",
      "INFO:tensorflow:global_step/sec: 32.9301\n",
      "INFO:tensorflow:loss = 0.104578935, step = 41901 (3.037 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0002\n",
      "INFO:tensorflow:loss = 0.1325797, step = 42001 (2.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0274\n",
      "INFO:tensorflow:loss = 0.051864635, step = 42101 (2.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2424\n",
      "INFO:tensorflow:loss = 0.05588101, step = 42201 (2.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0529\n",
      "INFO:tensorflow:loss = 0.022367716, step = 42301 (2.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0555\n",
      "INFO:tensorflow:loss = 0.049224246, step = 42401 (2.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2921\n",
      "INFO:tensorflow:loss = 0.03957125, step = 42501 (2.755 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0016\n",
      "INFO:tensorflow:loss = 0.07929339, step = 42601 (2.778 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0744\n",
      "INFO:tensorflow:loss = 0.03995199, step = 42701 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2336\n",
      "INFO:tensorflow:loss = 0.038024187, step = 42801 (2.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.1441\n",
      "INFO:tensorflow:loss = 0.024303207, step = 42901 (2.767 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.0707\n",
      "INFO:tensorflow:loss = 0.06355703, step = 43001 (2.772 sec)\n",
      "INFO:tensorflow:global_step/sec: 36.2503\n",
      "INFO:tensorflow:loss = 0.045988366, step = 43101 (2.759 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.4581\n",
      "INFO:tensorflow:loss = 0.029739289, step = 43201 (2.820 sec)\n",
      "INFO:tensorflow:global_step/sec: 35.3083\n",
      "INFO:tensorflow:loss = 0.047908597, step = 43301 (2.833 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9687\n",
      "INFO:tensorflow:loss = 0.043448128, step = 43401 (2.943 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.5293\n",
      "INFO:tensorflow:loss = 0.08201373, step = 43501 (2.984 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9542\n",
      "INFO:tensorflow:loss = 0.038880583, step = 43601 (2.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9604\n",
      "INFO:tensorflow:loss = 0.045643073, step = 43701 (2.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.088\n",
      "INFO:tensorflow:loss = 0.050600957, step = 43801 (2.934 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8945\n",
      "INFO:tensorflow:loss = 0.061200768, step = 43901 (2.951 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9596\n",
      "INFO:tensorflow:loss = 0.05764186, step = 44001 (2.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.7751\n",
      "INFO:tensorflow:loss = 0.028632421, step = 44101 (2.961 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9727\n",
      "INFO:tensorflow:loss = 0.023098333, step = 44201 (2.943 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0833\n",
      "INFO:tensorflow:loss = 0.045995574, step = 44301 (2.934 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0667\n",
      "INFO:tensorflow:loss = 0.04743005, step = 44401 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0133\n",
      "INFO:tensorflow:loss = 0.012050536, step = 44501 (2.940 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1751\n",
      "INFO:tensorflow:loss = 0.04441673, step = 44601 (2.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0217\n",
      "INFO:tensorflow:loss = 0.08112452, step = 44701 (2.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0704\n",
      "INFO:tensorflow:loss = 0.031846236, step = 44801 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9418\n",
      "INFO:tensorflow:loss = 0.10877518, step = 44901 (2.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0622\n",
      "INFO:tensorflow:loss = 0.07479305, step = 45001 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8989\n",
      "INFO:tensorflow:loss = 0.09370167, step = 45101 (2.950 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.846\n",
      "INFO:tensorflow:loss = 0.059754495, step = 45201 (2.955 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0596\n",
      "INFO:tensorflow:loss = 0.019705998, step = 45301 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9538\n",
      "INFO:tensorflow:loss = 0.05792666, step = 45401 (2.945 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0423\n",
      "INFO:tensorflow:loss = 0.04920329, step = 45501 (2.938 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0209\n",
      "INFO:tensorflow:loss = 0.014683869, step = 45601 (2.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0703\n",
      "INFO:tensorflow:loss = 0.028593378, step = 45701 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0577\n",
      "INFO:tensorflow:loss = 0.13944629, step = 45801 (2.937 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9181\n",
      "INFO:tensorflow:loss = 0.17683052, step = 45901 (2.948 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9252\n",
      "INFO:tensorflow:loss = 0.015655614, step = 46001 (2.948 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0328\n",
      "INFO:tensorflow:loss = 0.017784642, step = 46101 (2.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0947\n",
      "INFO:tensorflow:loss = 0.06648169, step = 46201 (2.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2229\n",
      "INFO:tensorflow:loss = 0.06334125, step = 46301 (2.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1027\n",
      "INFO:tensorflow:loss = 0.011765928, step = 46401 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0042\n",
      "INFO:tensorflow:loss = 0.066537306, step = 46501 (2.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1673\n",
      "INFO:tensorflow:loss = 0.052383274, step = 46601 (2.927 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.967\n",
      "INFO:tensorflow:loss = 0.044061996, step = 46701 (2.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9219\n",
      "INFO:tensorflow:loss = 0.033434525, step = 46801 (2.948 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1139\n",
      "INFO:tensorflow:loss = 0.029835394, step = 46901 (2.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0597\n",
      "INFO:tensorflow:loss = 0.037956074, step = 47001 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0148\n",
      "INFO:tensorflow:loss = 0.06701789, step = 47101 (2.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9856\n",
      "INFO:tensorflow:loss = 0.019898992, step = 47201 (2.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8664\n",
      "INFO:tensorflow:loss = 0.06480648, step = 47301 (2.948 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9262\n",
      "INFO:tensorflow:loss = 0.045658045, step = 47401 (2.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0345\n",
      "INFO:tensorflow:loss = 0.01939772, step = 47501 (2.939 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8408\n",
      "INFO:tensorflow:loss = 0.067222655, step = 47601 (2.955 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8935\n",
      "INFO:tensorflow:loss = 0.09853414, step = 47701 (2.952 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9149\n",
      "INFO:tensorflow:loss = 0.015305179, step = 47801 (2.948 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9831\n",
      "INFO:tensorflow:loss = 0.006404053, step = 47901 (2.943 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.881\n",
      "INFO:tensorflow:loss = 0.08851406, step = 48001 (2.951 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.134\n",
      "INFO:tensorflow:loss = 0.04138766, step = 48101 (2.929 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0006\n",
      "INFO:tensorflow:loss = 0.035975028, step = 48201 (2.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1426\n",
      "INFO:tensorflow:loss = 0.043141376, step = 48301 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.06208189, step = 48401 (2.940 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1238\n",
      "INFO:tensorflow:loss = 0.029821046, step = 48501 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1335\n",
      "INFO:tensorflow:loss = 0.036645398, step = 48601 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0648\n",
      "INFO:tensorflow:loss = 0.012435116, step = 48701 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0135\n",
      "INFO:tensorflow:loss = 0.028573735, step = 48801 (2.940 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1386\n",
      "INFO:tensorflow:loss = 0.017125783, step = 48901 (2.929 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1812\n",
      "INFO:tensorflow:loss = 0.023259396, step = 49001 (2.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0303\n",
      "INFO:tensorflow:loss = 0.027137034, step = 49101 (2.938 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2034\n",
      "INFO:tensorflow:loss = 0.020385506, step = 49201 (2.924 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0975\n",
      "INFO:tensorflow:loss = 0.09328075, step = 49301 (2.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0375\n",
      "INFO:tensorflow:loss = 0.07877976, step = 49401 (2.938 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0426\n",
      "INFO:tensorflow:loss = 0.083604306, step = 49501 (2.938 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1397\n",
      "INFO:tensorflow:loss = 0.026213916, step = 49601 (2.929 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1811\n",
      "INFO:tensorflow:loss = 0.012812856, step = 49701 (2.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0018\n",
      "INFO:tensorflow:loss = 0.09620248, step = 49801 (2.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9121\n",
      "INFO:tensorflow:loss = 0.036060926, step = 49901 (2.948 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.859\n",
      "INFO:tensorflow:loss = 0.014532704, step = 50001 (2.953 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8478\n",
      "INFO:tensorflow:loss = 0.011133742, step = 50101 (2.955 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9049\n",
      "INFO:tensorflow:loss = 0.096341364, step = 50201 (2.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0506\n",
      "INFO:tensorflow:loss = 0.008778788, step = 50301 (2.937 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1441\n",
      "INFO:tensorflow:loss = 0.022080975, step = 50401 (2.928 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9886\n",
      "INFO:tensorflow:loss = 0.010019461, step = 50501 (2.943 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0706\n",
      "INFO:tensorflow:loss = 0.04351592, step = 50601 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1268\n",
      "INFO:tensorflow:loss = 0.049824294, step = 50701 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0624\n",
      "INFO:tensorflow:loss = 0.037708297, step = 50801 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1345\n",
      "INFO:tensorflow:loss = 0.018548142, step = 50901 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1738\n",
      "INFO:tensorflow:loss = 0.008136616, step = 51001 (2.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9842\n",
      "INFO:tensorflow:loss = 0.034694582, step = 51101 (2.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1297\n",
      "INFO:tensorflow:loss = 0.017119803, step = 51201 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0841\n",
      "INFO:tensorflow:loss = 0.041625913, step = 51301 (2.934 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9719\n",
      "INFO:tensorflow:loss = 0.014294125, step = 51401 (2.943 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1817\n",
      "INFO:tensorflow:loss = 0.02244147, step = 51501 (2.926 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2201\n",
      "INFO:tensorflow:loss = 0.050503973, step = 51601 (2.922 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0405\n",
      "INFO:tensorflow:loss = 0.039455827, step = 51701 (2.938 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0002\n",
      "INFO:tensorflow:loss = 0.019553876, step = 51801 (2.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9635\n",
      "INFO:tensorflow:loss = 0.05068758, step = 51901 (2.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0437\n",
      "INFO:tensorflow:loss = 0.033273954, step = 52001 (2.937 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9586\n",
      "INFO:tensorflow:loss = 0.005693061, step = 52101 (2.945 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9409\n",
      "INFO:tensorflow:loss = 0.03665103, step = 52201 (2.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0103\n",
      "INFO:tensorflow:loss = 0.02388824, step = 52301 (2.940 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1477\n",
      "INFO:tensorflow:loss = 0.046274997, step = 52401 (2.928 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8962\n",
      "INFO:tensorflow:loss = 0.14323649, step = 52501 (2.951 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9376\n",
      "INFO:tensorflow:loss = 0.021938683, step = 52601 (2.946 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9083\n",
      "INFO:tensorflow:loss = 0.014466879, step = 52701 (2.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0532\n",
      "INFO:tensorflow:loss = 0.07330909, step = 52801 (2.937 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0785\n",
      "INFO:tensorflow:loss = 0.06585437, step = 52901 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1181\n",
      "INFO:tensorflow:loss = 0.02489441, step = 53001 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8342\n",
      "INFO:tensorflow:loss = 0.007723486, step = 53101 (2.957 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.978\n",
      "INFO:tensorflow:loss = 0.017641852, step = 53201 (2.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2001\n",
      "INFO:tensorflow:loss = 0.0686623, step = 53301 (2.925 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9277\n",
      "INFO:tensorflow:loss = 0.04478329, step = 53401 (2.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.136\n",
      "INFO:tensorflow:loss = 0.014677921, step = 53501 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1121\n",
      "INFO:tensorflow:loss = 0.08302515, step = 53601 (2.931 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1206\n",
      "INFO:tensorflow:loss = 0.01331182, step = 53701 (2.931 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8879\n",
      "INFO:tensorflow:loss = 0.04407271, step = 53801 (2.950 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.106\n",
      "INFO:tensorflow:loss = 0.075602785, step = 53901 (2.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8978\n",
      "INFO:tensorflow:loss = 0.0188157, step = 54001 (2.951 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1142\n",
      "INFO:tensorflow:loss = 0.013551354, step = 54101 (2.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0608\n",
      "INFO:tensorflow:loss = 0.018512577, step = 54201 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0619\n",
      "INFO:tensorflow:loss = 0.055022, step = 54301 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2414\n",
      "INFO:tensorflow:loss = 0.029391648, step = 54401 (2.920 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2387\n",
      "INFO:tensorflow:loss = 0.057788204, step = 54501 (2.923 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.122\n",
      "INFO:tensorflow:loss = 0.03445045, step = 54601 (2.929 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.181\n",
      "INFO:tensorflow:loss = 0.045214556, step = 54701 (2.925 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9844\n",
      "INFO:tensorflow:loss = 0.00876331, step = 54801 (2.942 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2504\n",
      "INFO:tensorflow:loss = 0.024871288, step = 54901 (2.920 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 55000 into /tmp/mnist_vgg13_model/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.007031409.\n",
      "INFO:tensorflow:Starting evaluation at 2018-02-22-01:01:23\n",
      "INFO:tensorflow:Restoring parameters from /tmp/mnist_vgg13_model/model.ckpt-55000\n",
      "INFO:tensorflow:Finished evaluation at 2018-02-22-01:01:24\n",
      "INFO:tensorflow:Saving dict for global step 55000: accuracy = 0.9842, global_step = 55000, loss = 0.055481646\n",
      "{'accuracy': 0.9842, 'loss': 0.055481646, 'global_step': 55000}\n"
     ]
    }
   ],
   "source": [
    "# Create the Estimator\n",
    "mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn,\n",
    "                                          model_dir=\"/tmp/mnist_vgg13_model\")\n",
    "\n",
    "# Train the model\n",
    "train_input_fn = tf.estimator.inputs.numpy_input_fn(x={\"x\": train_data},\n",
    "                                                    y=train_labels,\n",
    "                                                    batch_size=100,\n",
    "                                                    num_epochs=100,\n",
    "                                                    shuffle=True)\n",
    "mnist_classifier.train(input_fn=train_input_fn,\n",
    "                       steps=None,\n",
    "                       hooks=None)\n",
    "\n",
    "# Evaluate the model and print results\n",
    "eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={\"x\": eval_data},\n",
    "                                                   y=eval_labels,\n",
    "                                                   num_epochs=1,\n",
    "                                                   shuffle=False)\n",
    "eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)\n",
    "print(eval_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "coursera": {
   "course_slug": "convolutional-neural-networks",
   "graded_item_id": "bwbJV",
   "launcher_item_id": "0TkXB"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
